﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;

using QUT.Gppg;

using Microsoft.VisualStudio.Package;
using Microsoft.VisualStudio.TextManager.Interop;

namespace BlunderBrain.VisualStudio.AviSynth
{
    /// <summary>
    /// Location type used for token tracking
    /// </summary>
    public class TokenSpan : IMerge<TokenSpan>
    {
        public TextSpan Span;

        public TokenSpan() : this(0, 0, 0, 0) { }

        public TokenSpan(int startLine, int startColumn, int endLine, int endColumn)
        {
            Span.iStartLine = startLine;
            Span.iStartIndex = startColumn;
            Span.iEndLine = endLine;
            Span.iEndIndex = endColumn;
        }

        public TokenSpan Merge(TokenSpan last)
        {
            return new TokenSpan(this.Span.iStartLine, this.Span.iStartIndex, last.Span.iEndLine, last.Span.iEndIndex);
        }
    }


    public class AviSynthTokenizer : AbstractScanner<TokenValue, TokenSpan>, IScanner
    {
        private const int STATE_DEFAULT = 0;
        private const int STATE_IN_BLOCKCOMMENT = 0x00010000;
        private const int STATE_IN_NESTED_BLOCKCOMMENT = 0x00020000;
        private const int STATE_END_OF_SCRIPT = 0x00040000;

        private string m_source;
        private int m_pc;
        private TokenInfo m_tokenInfo;
        private int m_state = STATE_DEFAULT;
        private int m_nestedCommentDepth = 0;
        private int m_currentLine = 1;     // Current line in the source that's being parsed
        private int m_currentLineOffset = 0;   // Offset in the source to the start of the current line
        private TokenSpan m_yylloc;

        private StringBuilder m_stringBuffer = new StringBuilder();
        private object m_syncObj = new object();

        private Regex m_singleQuotedStringParser = new Regex("^(\"[^\"]*\")", RegexOptions.Multiline);
        private Regex m_trippleQuotedStringParser = new Regex("^(\"{3}.*(?<!\"{3})\"{3})|^\"{6}", RegexOptions.Multiline);  // """ followed by anything that does not contain """ followed by """.  OR """"""

        private Regex m_statementSplitEnd = new Regex(@"\\\s*(?:(?:\r?\n)|\z)");
        private Regex m_statementSplitStart = new Regex(@"\A\r*\n\s*\\");

        public static string[] Keywords { get; private set; }

        static AviSynthTokenizer()
        {
            Keywords = new string[] {"clip", "int", "float", "string", "bool", "val", "true", "false", "yes", "no", "function", "global", "return", "try", "catch", "__END__" };
        }

        public AviSynthTokenizer()
        {
            m_tokenInfo = new TokenInfo();  // Used when we call ScanTokenAndProvideInfoAboutIt() via yylex()
            m_yylloc = new TokenSpan();
        }

        public string Source
        {
            get { return m_source; }
        }

        public override TokenSpan yylloc
        {
            get
            {
                return m_yylloc;
            }
            set
            {
                m_yylloc = value;
            }
        }

        /// <summary>
        /// Gets or sets the parse request that triggered the current parsing operation. This object is used to report parsing errors.
        /// </summary>
        public AviSynthParseRequest ParseRequest { get; set; }


        public string GetTokenValue(TokenInfo token)
        {
            return m_source.Substring(token.StartIndex, token.EndIndex - token.StartIndex + 1);
        }

        public bool ScanTokenAndProvideInfoAboutIt(TokenInfo tokenInfo, ref int state)
        {
            lock (m_syncObj)
            {
                if (state == STATE_END_OF_SCRIPT)   // Found __END__ on previous call, abort
                    return false;
                
                m_tokenInfo = tokenInfo;

                // TODO: state handling to support multiline strings in line scanning mode

                if ((state & STATE_IN_NESTED_BLOCKCOMMENT) != 0)    // For nested blockcomments the lower 16-bit of the state variable indicates our current depth
                {
                    m_nestedCommentDepth = state & 0xFFFF;
                    m_state = STATE_IN_NESTED_BLOCKCOMMENT;
                }
                else
                {
                    m_nestedCommentDepth = 0;
                    m_state = state;
                }

                bool tokenFound = GetNextToken();

                if (m_state == STATE_IN_NESTED_BLOCKCOMMENT)
                    state = STATE_IN_NESTED_BLOCKCOMMENT | m_nestedCommentDepth;
                else
                    state = m_state;

                return tokenFound;
            }
        }

        public void SetSource(string source, int offset)
        {
            lock (m_syncObj)
            {
                m_source = source;
                m_pc = offset;
                m_currentLine = 0;
                m_currentLineOffset = offset;
            }
        }

        private bool GetNextToken()
        {
            bool tokenFound = false;
            int tokenStart = m_pc;

            if (string.IsNullOrEmpty(m_source) || m_pc >= m_source.Length)
                return false;

            do
            {
                if (IsBlockCommentStart || m_state == STATE_IN_BLOCKCOMMENT)
                {
                    tokenStart = m_pc;
                    ReadBlockComment();

                    SetToComment(tokenStart, m_pc);
                    tokenFound = true;
                    break;
                }
                else if (IsNestedBlockCommentStart || m_state == STATE_IN_NESTED_BLOCKCOMMENT)
                {
                    tokenStart = m_pc;
                    ReadNestedBlockComment();

                    SetToComment(tokenStart, m_pc);
                    tokenFound = true;
                    break;
                }
                else if (IsLineCommentStart)     // single line comments
                {
                    tokenStart = m_pc;
                    ReadLineComment();
                    SetToComment(tokenStart, m_pc);
                    tokenFound = true;
                    break;
                }

                switch (m_source[m_pc])
                {
                    case '\\':      // Handle statement splits at end of line
                        SetTokenStart();
                        tokenStart = m_pc;
                        Match m1 = m_statementSplitEnd.Match(m_source.Substring(m_pc));
                        if (m1.Success)     // Found a valid split?
                        {
                            m_currentLine++;
                            m_currentLineOffset = m_pc + m1.Value.IndexOf('\n') + 1;
                            m_pc += m1.Length - 1;  // Eat the '\' + following whitespace and newline
                        }
                        else
                            m_pc++;     // Not a valid split, just eat the '\' and let the parser blow up

                        SetToWhiteSpace(tokenStart, m_pc);
                        SetTokenEnd();
                        tokenFound = true;
                        break;
                    case '\n':
                        SetTokenStart();
                        m_currentLine++;
                        m_currentLineOffset = m_pc + 1;
                        // Check if the following line contains a statement split, and if so eat the newline + split
                        Match m2 = m_statementSplitStart.Match(m_source.Substring(m_pc)); 
                        if (m2.Success)
                        {
                            tokenStart = m_pc;
                            m_pc += m2.Length - 1;
                            SetToWhiteSpace(tokenStart, m_pc);
                        }
                        else
                            SetToDelimeter(m_pc, m_pc);     // No statement split found, treat the newline as a statement terminator
                        SetTokenEnd();
                        tokenFound = true;
                        break;
                    case ' ':
                    case '\t':
                    case '\r':
                        SetTokenStart();
                        tokenStart = m_pc;
                        ReadWhiteSpace();
                        SetToWhiteSpace(tokenStart, m_pc);
                        SetTokenEnd();
                        tokenFound = true;
                        break;
                    case '.':
                        SetTokenStart();
                        tokenStart = m_pc;
                        if (!IsEndOfBuffer)
                            m_pc++;
                        if (IsNumeric)
                        {
                            ReadNumber();
                            SetToNumber(tokenStart, m_pc);
                        }
                        else
                        {
                            m_pc = tokenStart;
                            SetToOperator(m_pc, m_pc);
                        }
                        SetTokenEnd();
                        tokenFound = true;
                        break;
                    case '<':
                        SetTokenStart();
                        if (IsEndOfLine)
                            SetToOperator(m_pc, m_pc);
                        else if (m_source[m_pc + 1] == '>' || m_source[m_pc + 1] == '=')
                            SetToOperator(m_pc, ++m_pc);
                        else
                            SetToOperator(m_pc, m_pc);

                        SetTokenEnd();
                        tokenFound = true;
                        break;
                    case '>':
                    case '!':
                    case '=':
                        SetTokenStart();
                        if (IsEndOfBuffer)
                            SetToOperator(m_pc, m_pc);
                        else if (m_source[m_pc + 1] == '=')
                            SetToOperator(m_pc, ++m_pc);
                        else
                            SetToOperator(m_pc, m_pc);

                        SetTokenEnd();
                        tokenFound = true;
                        break;
                    case '+':    // these operators have single and double (++, &&, ||, ==) versions
                    case '&':
                    case '|':
                        SetTokenStart();
                        if (IsEndOfLine)
                            SetToOperator(m_pc, m_pc);
                        else if (m_source[m_pc] == m_source[m_pc + 1])
                            SetToOperator(m_pc, ++m_pc);
                        else
                            SetToOperator(m_pc, m_pc);

                        SetTokenEnd();
                        tokenFound = true;
                        break;
                    case '{':
                    case '}':
                    case '(':
                    case ')':
                    case ',':
                        SetTokenStart();
                        SetToDelimeter(m_pc, m_pc);
                        SetTokenEnd();
                        tokenFound = true;
                        break;
                    case '?':
                    case ':':
                    case '-':
                    case '*':
                    case '/':
                    case '%':
                        SetTokenStart();
                        SetToOperator(m_pc, m_pc);
                        SetTokenEnd();
                        tokenFound = true;    
                        break;
                    case '$':
                        SetTokenStart();
                        tokenStart = m_pc;
                        ReadHexNumber();
                        SetToNumber(tokenStart, m_pc);
                        SetTokenEnd();
                        tokenFound = true;
                        break;
                    case '"':
                        SetTokenStart();
                        tokenStart = m_pc;
                        ReadString();
                        SetToString(tokenStart, m_pc);
                        SetTokenEnd();
                        tokenFound = true;
                        break;
                    default:
                        SetTokenStart();
                        tokenStart = m_pc;
                        if (IsNumeric)
                        {
                            ReadNumber();
                            SetToNumber(tokenStart, m_pc);
                            SetTokenEnd();
                            tokenFound = true;
                        }
                        else if (IsAlphaNumeric)
                        {
                            string identifier = ReadIdentifier();
                            if (IsKeyword(identifier))
                                SetToKeyword(tokenStart, m_pc);
                            else
                                SetToIdentifier(tokenStart, m_pc);

                            SetTokenEnd();
                            tokenFound = true;
                        }
                        else
                            m_pc++; // Unknown token, ignore
                        break;
                }

            } while (!IsEndOfBuffer && !tokenFound);

            m_pc++;

            return tokenFound;
        }

        #region Identificators
        private bool IsEndOfBuffer
        {
            get { return m_pc >= m_source.Length - 1; }
        }

        private bool IsEndOfLine
        {
            get
            {
                if (IsEndOfBuffer)
                    return true;
                else if (m_source[m_pc + 1] == '\n')
                    return true;
                else
                    return false;
            }
        }

        private bool IsLineCommentStart
        {
            get { return m_source[m_pc] == '#'; }
        }

        private bool IsBlockCommentStart
        {
            get { return IsEndOfBuffer ? false : (m_source[m_pc] == '/' && m_source[m_pc + 1] == '*'); }
        }

        private bool IsBlockCommentEnd
        {
            get { return IsEndOfBuffer ? false : (m_source[m_pc] == '*' && m_source[m_pc + 1] == '/'); }
        }

        private bool IsNestedBlockCommentStart
        {
            get { return IsEndOfBuffer ? false : (m_source[m_pc] == '[' && m_source[m_pc + 1] == '*'); }
        }

        private bool IsNestedBlockCommentEnd
        {
            get { return IsEndOfBuffer ? false : (m_source[m_pc] == '*' && m_source[m_pc + 1] == ']'); }
        }

        private bool IsWhiteSpace
        {
            // IsWhiteSpace() includes newlines, but since '\n' is used as a statement terminator, filter it out
            get { return char.IsWhiteSpace(m_source[m_pc]) && m_source[m_pc] != '\n'; }
        }

        private bool IsNumeric
        {
            get { return char.IsNumber(m_source[m_pc]); }
        }

        private bool IsHexNumeric
        {
            get
            {
                return (
                    (m_source[m_pc] == '$') || char.IsDigit(m_source[m_pc]) ||
                    (m_source[m_pc] >= 'a' && m_source[m_pc] <= 'f') ||
                    (m_source[m_pc] >= 'A' && m_source[m_pc] <= 'F'));
            }
        }

        private bool IsAlphaNumeric
        {
            get { return (m_source[m_pc] == '_') || char.IsLetterOrDigit(m_source[m_pc]); }
        }

        private bool IsKeyword(string identifier)
        {
            return AviSynthTokenizer.Keywords.Any(keyword => string.Compare(keyword, identifier, true) == 0);
        }
        #endregion

        private void CheckAndIncrementLineCount()
        {
            if (m_source[m_pc] == '\n')
            {
                m_currentLine++;
                m_currentLineOffset = m_pc + 1;
            }
        }

        #region Token read functions
        
        private void ReadLineComment()
        {
            while (!IsEndOfLine)  // Scan until end of buffer or newline
            {
                m_pc++;
                CheckAndIncrementLineCount();
            }
        }

        private void ReadBlockComment()
        {
            while (!IsEndOfBuffer && !IsBlockCommentEnd)
            {
                m_pc++;
                CheckAndIncrementLineCount();
            }

            if (IsBlockCommentEnd)
            {
                m_state = STATE_DEFAULT;
                m_pc++;
            }
            else
                m_state = STATE_IN_BLOCKCOMMENT;
        }

        private void ReadNestedBlockComment()
        {
            if (IsNestedBlockCommentStart)
                m_nestedCommentDepth++;

            while (!IsEndOfBuffer && m_nestedCommentDepth > 0)
            {
                m_pc++;
                if (IsNestedBlockCommentStart)
                    ReadNestedBlockComment();
                else if (IsNestedBlockCommentEnd)
                    m_nestedCommentDepth--;
                else
                    CheckAndIncrementLineCount();
            }

            m_state = m_nestedCommentDepth == 0 ? STATE_DEFAULT : STATE_IN_NESTED_BLOCKCOMMENT;

            if (!IsEndOfBuffer)
                m_pc++;
        }

        private void ReadWhiteSpace()
        {
            while (!IsEndOfBuffer && IsWhiteSpace)
                m_pc++;

            if (!IsEndOfBuffer || (IsEndOfBuffer && !IsWhiteSpace))
                m_pc--;
        }

        private void ReadNumber()
        {
            while (!IsEndOfBuffer && ( IsNumeric || m_source[m_pc] == '.' ))
                m_pc++;

            if (!IsEndOfBuffer || ( IsEndOfBuffer && !IsNumeric))
                m_pc--;
        }

        private void ReadHexNumber()
        {
            while (!IsEndOfBuffer && IsHexNumeric)
                m_pc++;

            if (!IsEndOfBuffer || (IsEndOfBuffer && !IsHexNumeric))
                m_pc--;
        }

        private void ReadString()
        {
            string substring = m_source.Substring(m_pc);

            Match m = m_trippleQuotedStringParser.Match(substring);
            
            if (!m.Success)
                m = m_singleQuotedStringParser.Match(substring);

            if (m.Success)
            {
                m_currentLine += m.Value.Count(c => c == '\n');
                m_currentLineOffset = m_pc + substring.LastIndexOf('\n') + 1;

                m_pc += m.Length - 1;
            }
            else    // Couldn't find end of string, treat rest of buffer as part of string
            {
                m_currentLine += substring.Count(c => c == '\n');
                m_currentLineOffset = m_source.Length - 1;
                m_pc = m_source.Length - 1;
            }
        }

        private string ReadIdentifier()
        {
            m_stringBuffer.Length = 0;
            while (!IsEndOfBuffer && IsAlphaNumeric)
            {
                m_stringBuffer.Append(m_source[m_pc]);
                m_pc++;
            }
            if (IsAlphaNumeric)
                m_stringBuffer.Append(m_source[m_pc]);

            if (!IsEndOfBuffer || (IsEndOfBuffer && !IsAlphaNumeric))
                m_pc--;

            return m_stringBuffer.ToString();
        }
#endregion

        #region Token set functions

        private void SetTokenStart()
        {
            yylloc.Span.iStartLine = m_currentLine;
            yylloc.Span.iStartIndex = m_pc - Math.Min(m_currentLineOffset, m_source.Length - 1);
        }

        private void SetTokenEnd()
        {
            yylloc.Span.iEndLine = m_currentLine;
            yylloc.Span.iEndIndex = m_pc - Math.Min(m_currentLineOffset, m_source.Length - 1);
        }

        private void SetToComment(int startIndex, int endIndex)
        {
            m_tokenInfo.Type = TokenType.Comment;
            m_tokenInfo.Color = TokenColor.Comment;
            m_tokenInfo.StartIndex = startIndex;
            m_tokenInfo.EndIndex = endIndex;
            m_tokenInfo.Trigger = TokenTriggers.None;
        }

        private void SetToWhiteSpace(int startIndex, int endIndex)
        {
            m_tokenInfo.Type = TokenType.WhiteSpace;
            m_tokenInfo.Color = TokenColor.Text;
            m_tokenInfo.StartIndex = startIndex;
            m_tokenInfo.EndIndex = endIndex;
            m_tokenInfo.Trigger = TokenTriggers.None;
        }

        private void SetToNumber(int startIndex, int endIndex)
        {
            m_tokenInfo.Type = TokenType.Literal;
            m_tokenInfo.Color = TokenColor.Number;
            m_tokenInfo.StartIndex = startIndex;
            m_tokenInfo.EndIndex = endIndex;
            m_tokenInfo.Trigger = TokenTriggers.None;

            string token = GetTokenValue(m_tokenInfo);
            if (token.Contains('$'))
                m_tokenInfo.Token = (int)AviSynthTokens.HEX;
            else if (token.Contains('.'))
                m_tokenInfo.Token = (int)AviSynthTokens.FLOAT;
            else
                m_tokenInfo.Token = (int)AviSynthTokens.INTEGER;

            yylval = new TokenValue(m_tokenInfo);
        }

        private void SetToString(int startIndex, int endIndex)
        {
            m_tokenInfo.Type = TokenType.String;
            m_tokenInfo.Color = TokenColor.String;
            m_tokenInfo.StartIndex = startIndex;
            m_tokenInfo.EndIndex = endIndex;
            m_tokenInfo.Trigger = TokenTriggers.None;

            m_tokenInfo.Token = (int)AviSynthTokens.STRINGLITERAL;

            yylval = new TokenValue(m_tokenInfo);
        }

        private void SetToOperator(int startIndex, int endIndex)
        {
            m_tokenInfo.Type = TokenType.Operator;
            m_tokenInfo.Color = TokenColor.Text;
            m_tokenInfo.StartIndex = startIndex;
            m_tokenInfo.EndIndex = endIndex;
            m_tokenInfo.Trigger = TokenTriggers.None;

            string token = GetTokenValue(m_tokenInfo);
            switch (token)
            {
                case "==":
                case "!=":
                case "<>":
                    m_tokenInfo.Token = (int)AviSynthTokens.EQUALITY_BINARY_OP;
                    break;
                case "<":
                case ">":
                case "<=":
                case ">=":
                    m_tokenInfo.Token = (int)AviSynthTokens.RELATIONAL_BINARY_OP;
                    break;
                case "++":
                    m_tokenInfo.Token = (int)AviSynthTokens.ALIGNED_SPLICE_OP;
                    break;
                case "&&":
                    m_tokenInfo.Token = (int)AviSynthTokens.LOGICAL_AND_OP;
                    break;
                case "||":
                    m_tokenInfo.Token = (int)AviSynthTokens.LOGICAL_OR_OP;
                    break;
                default:
                    m_tokenInfo.Token = (int)token[0];
                    break;
            }

            yylval = new TokenValue(m_tokenInfo);
        }

        private void SetToDelimeter(int startIndex, int endIndex)
        {
            m_tokenInfo.Type = TokenType.Delimiter;
            m_tokenInfo.Color = TokenColor.Text;
            m_tokenInfo.StartIndex = startIndex;
            m_tokenInfo.EndIndex = endIndex;

            string token = GetTokenValue(m_tokenInfo);
            m_tokenInfo.Token = (int)token[0];

            switch (token)
            {
                case "(":
                    m_tokenInfo.Trigger = TokenTriggers.ParameterStart;
                    break;
                case ")":
                    m_tokenInfo.Trigger = TokenTriggers.ParameterEnd;
                    break;
                case ",":
                    m_tokenInfo.Trigger = TokenTriggers.ParameterNext;
                    break;
                case "{":
                    m_tokenInfo.Token = (int)AviSynthTokens.COMPOUND_START;
                    break;
                case "}":
                    m_tokenInfo.Token = (int)AviSynthTokens.COMPOUND_END;
                    break;
                default:
                    m_tokenInfo.Trigger = TokenTriggers.None;
                    break;
            }

            
            yylval = new TokenValue(m_tokenInfo);
        }

        private void SetToIdentifier(int startIndex, int endIndex)
        {
            m_tokenInfo.Type = TokenType.Identifier;
            m_tokenInfo.Color = TokenColor.Identifier;
            m_tokenInfo.StartIndex = startIndex;
            m_tokenInfo.EndIndex = endIndex;
            m_tokenInfo.Trigger = TokenTriggers.None;

            m_tokenInfo.Token = (int)AviSynthTokens.IDENTIFIER;
            yylval = new TokenValue(m_tokenInfo, GetTokenValue(m_tokenInfo));
        }

        private void SetToKeyword(int startIndex, int endIndex)
        {
            m_tokenInfo.Type = TokenType.Keyword;
            m_tokenInfo.Color = TokenColor.Keyword;
            m_tokenInfo.StartIndex = startIndex;
            m_tokenInfo.EndIndex = endIndex;
            m_tokenInfo.Trigger = TokenTriggers.None;

            string token = GetTokenValue(m_tokenInfo);
            switch (token.ToLower())
            {
                case "clip":
                    m_tokenInfo.Token = (int)AviSynthTokens.T_CLIP;
                    break;
                case "int":
                    m_tokenInfo.Token = (int)AviSynthTokens.T_INT;
                    break;
                case "float":
                    m_tokenInfo.Token = (int)AviSynthTokens.T_FLOAT;
                    break;
                case "string":
                    m_tokenInfo.Token = (int)AviSynthTokens.T_STRING;
                    break;
                case "bool":
                    m_tokenInfo.Token = (int)AviSynthTokens.T_BOOL;
                    break;
                case "val":
                    m_tokenInfo.Token = (int)AviSynthTokens.T_VAL;
                    break;
                case "true":
                case "false":
                case "yes":
                case "no":
                    m_tokenInfo.Token = (int)AviSynthTokens.BOOLEAN;
                    break;
                case "function":
                    m_tokenInfo.Token = (int)AviSynthTokens.KW_FUNCTION;
                    break;
                case "global":
                    m_tokenInfo.Token = (int)AviSynthTokens.KW_GLOBAL;
                    break;
                case "return":
                    m_tokenInfo.Token = (int)AviSynthTokens.KW_RETURN;
                    break;
                case "try":
                    m_tokenInfo.Token = (int)AviSynthTokens.KW_TRY;
                    break;
                case "catch":
                    m_tokenInfo.Token = (int)AviSynthTokens.KW_CATCH;
                    break;
                case "__END__":
                    m_tokenInfo.Token = (int)AviSynthTokens.KW_END;
                    m_state = STATE_END_OF_SCRIPT;
                    break;
                default:
                    m_tokenInfo.Token = (int)AviSynthTokens.IDENTIFIER;
                    break;
            }

            yylval = new TokenValue(m_tokenInfo, token);
        }
        #endregion

        #region AbstractScanner methods

        public override int yylex()
        {
            m_tokenInfo.Type = TokenType.Unknown;
            do
            {
                if (!ScanTokenAndProvideInfoAboutIt(m_tokenInfo, ref m_state))
                {
                    m_tokenInfo.Token = (int)AviSynthTokens.EOF;
                    break;
                }

            } while (m_tokenInfo.Type == TokenType.Comment || m_tokenInfo.Type == TokenType.LineComment || m_tokenInfo.Type == TokenType.WhiteSpace);

            return m_tokenInfo.Token;
        }

        public override void yyerror(string format, params object[] args)
        {
            System.Diagnostics.Trace.WriteLine(string.Format(format, args));

            if (this.ParseRequest != null && yylloc != null)
            {
                this.ParseRequest.Sink.AddError(
                    this.ParseRequest.FileName,
                    string.Format(format, args),
                    yylloc.Span,
                    Severity.Error);
            }

            base.yyerror(format, args);
        }

        #endregion
    }
}
